package server;


import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import models.Domain;
import models.Inlink;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyService;

import controller.Utils;

public class UpdateDomain {
    
    private URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
    
    private Domain domain = new Domain();
    
    private TreeMap<String, String> check = new TreeMap<String, String>();
    
    private Future fDomain = null;
    
    private Future fWhois = null;
    
    private Future fIpAddress = null;
    
    private Future fAlexa = null;
    
    private Future fGoogleRank = null;
    
    private Future fAgeDomain = null;
    
    private Future fValidator = null;
    
    private Future fGoogleService = null;
    
    private Future fHeader = null;
    
    private Future fDns = null;
    
    private Future fTrace = null;

    private String content = "";
    
    private String alexaContent = "";
    
    public UpdateDomain(Domain domain) {
        this.domain = domain;
        //Load content from urls
        loadUrlDomain();
        
        loadUrlWhois();
        
        loadUrlIpAddress();
        
        loadUrlAlexa();
        
        loadUrlGoogleRank();
        
        loadUrlAgeDomain();
        
        loadHeader();
        
        loadDns();
        
        loadTrace();

        //Get content form url loaded
        fDomainService();
        
        getWhois();
        
        getIpAddress();
        
        fAlexaService();
        
        getGooglePageRank();
        
        getAgeDomain();
        
        getHeader();
        
        getDns();
        
        getTrace();
        
        try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		Objectify objectify = ObjectifyService.begin();
		
		this.domain.setDate(new Date());
		PutCache.putCacheList(this.domain.getDomain(), "updates",19);
		
		objectify.put(this.domain);
    }    

    /************************************* Load URL *************************************/
    
    private void loadHeader() {
        try {
            URL urlHeader = new URL("http://network-tools.com/default.asp?prog=httphead&host="+domain.getDomain());
            fHeader = fetcher.fetchAsync(urlHeader);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadDns() {
        try {
            URL urlDns = new URL("http://network-tools.com/default.asp?prog=dnsrec&host="+domain.getDomain());
            fDns = fetcher.fetchAsync(urlDns);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadTrace() {
        try {
            URL urlTrace = new URL("http://network-tools.com/default.asp?prog=trace&host="+domain.getDomain());
            fTrace = fetcher.fetchAsync(urlTrace);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadUrlDomain() {
        try {
            URL urlDomain = new URL("http://" + domain.getDomain());
            fDomain = fetcher.fetchAsync(urlDomain);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadUrlWhois() {
        try {
            URL urlWhois = new URL("http://www.whoismind.com/whois/"+domain.getDomain()+".html");
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Can't run fWhois");
        }
    }
    
    private void loadUrlIpAddress() {
        try {
            URL urlIpAddress = new URL("http://www.nearby.org.uk/api/dig.php?l=" + domain.getDomain());
            fIpAddress = fetcher.fetchAsync(urlIpAddress);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fIpAddress", "Can't run fIpAddress");
        }
    }
    
    private void loadUrlAlexa() {
        try {
            URL urlAlexa = new URL("http://www.alexa.com/siteinfo/" + domain.getDomain());
            fAlexa = fetcher.fetchAsync(urlAlexa);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't run fAlexa");
        }
    }
    
    private void loadUrlGoogleRank() {
        try {
            URL urlGoogleRank = new URL("http://gpageranks.appspot.com/?url=" + domain.getDomain());
            fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fGoogleRank", "Can't run fGoogleRank");
        }
    }
    
    private void loadUrlAgeDomain() {
        try {
            URL urlAgeDomain = new URL("http://www.seologs.com/dns/domain-check.html");
            HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            String params = "query=" + domain.getDomain() + "&output=cdt";
            request.setPayload(params.getBytes());
            fAgeDomain = fetcher.fetchAsync(request);

        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAgeDomain", "Can't run fAgeDomain");
        }
    }
    

    /************************************* Get content *************************************/
    private void fDomainService() {
        try {
        	HTTPResponse response = (HTTPResponse) fDomain.get();
            
            // if redirects are followed, this returns the final URL we are redirected to
            URL finalUrl = response.getFinalUrl();
            
            String charset="utf-8";
            
            this.content = new String(response.getContent(),charset);
    			
            Document doc = Jsoup.parse(this.content);
            
            int index = doc.select("meta").toString().toLowerCase()
			.indexOf("charset=");
			if (index != -1) {
				charset = doc.select("meta").toString().substring(index + 8)
						.replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "")
						.replaceAll("[\\'\\\"]+", "");
			}
            
			this.content = new String(response.getContent(),charset);
			
    		getServerInfo(response);
            
            getSiteInfo();
            getGoogleAnalytic();
            getGoogleAdsense();
            getJavascriptLibrary();
            getInlink();
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't response fDomain");
        }
    }
    
    private void fAlexaService() {
        try {
        	HTTPResponse response = (HTTPResponse) fAlexa.get();
            
            this.alexaContent = new String(response.getContent(),"utf-8");
            
            getAlexaRank();
            getPopularKeyword();
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getHeader() {
        try {
        	HTTPResponse response = (HTTPResponse) fHeader.get();
            
        	Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
        	Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
            this.domain.setHeader(new Text(elements.html()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getDns() {
        try {
        	HTTPResponse response = (HTTPResponse) fDns.get();
            
        	Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
        	Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
            this.domain.setDns(new Text(elements.html()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getTrace() {
        try {
        	HTTPResponse response = (HTTPResponse) fTrace.get();
            
        	Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
        	Element elements = doc.select("div[style=width:680px; float:left; margin-top:25px;]").get(0);
			elements.select("center").remove();
            this.domain.setTrace(new Text(elements.html()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getPopularKeyword() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements elements = doc.select("#top-keywords-from-search");
            String keyword = "";
            if(elements.size() > 0)
			{
				for (Element element : elements.first().select("u")) {
					keyword+=element.text()+", ";
				}
			}
            domain.setPopularKeyword(new Text(keyword));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    
    private void getServerInfo(HTTPResponse response) {
    	try {
            //Get webServer
            String webServer = "";
            String programmingLanguage = "";
            for (int i = 0; i < response.getHeaders().size(); i++) {
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("server")) {
                    webServer = response.getHeaders().get(i).getValue();
                }
                
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("x-powered-by")) {
                    programmingLanguage = response.getHeaders().get(i).getValue();
                }
                //System.out.println(response.getHeaders().get(i).getName() + " - " + response.getHeaders().get(i).getValue());
            }
            
            this.domain.setServer(webServer);
            this.domain.setLanguage(programmingLanguage);
            
        } catch (Exception e) {
            this.check.put("server", domain.getDomain() + " - Webserver not found");
        }
    }
    
    private void getSiteInfo() {
        try {
            Document doc = Jsoup.parse(this.content);
            
            String title = "";
            String description = "";
            String keywords  = "";
            String des = "";
            try {
                title = doc.select("title").first().text();
            } catch (Exception e) {
            	e.printStackTrace();
                this.check.put("title", "Can't get info title");
            }

            try {
                keywords = doc.select("meta[name=keywords]").first().attr("content");
            } catch (Exception e) {
                try {
                    keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
                } catch (Exception e2) {
                    this.check.put("keywords", "Can't get info keywords");
                }        
            }
            
            try {
                description = doc.select("meta[name=description]").first().attr("content");
            } catch (Exception e) {
                try {
                    description = doc.select("meta[http-equiv=description]").first().attr("content");
                } catch (Exception e2) {
                    if(doc.select("body").first().text().length() > 260)
                    {
                        description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
                    } else {
                        description = doc.select("body").first().text();
                    }
                }
            }
            des = doc.select("body p").text();
            if(des.length()<100)
            	des = doc.select("body").first().text();
            if(des.length()>250)
            	des = des.substring(0,240);
            
            
            if(description.length()>260)
            {
                description = description.substring(0,250);
            }
            
            this.domain.setTitle(title);
            this.domain.setKeywords(new Text(keywords));
            this.domain.setDescription(new Text(description));
            this.domain.setDes(new Text(des));
            
        } catch (Exception e) {
        	e.printStackTrace();
            this.check.put("siteinfo", "Can't connect to domain " + domain.getDomain());
        }
    }
    
    private void getGoogleAnalytic() {
        int begin = content.indexOf("_gaq.push(['_setAccount', '");
        String analyticAccount = "";
        if(begin>0)
        {
            begin+="_gaq.push(['_setAccount', '".length();
            int end =  content.indexOf("'",begin);
            analyticAccount = content.substring(begin, end);
            begin=analyticAccount.lastIndexOf("-")-1;
            
            analyticAccount = analyticAccount.substring(0, begin);
        }
        else
        {
             begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
            if(begin>0)
            {
                begin+="_gaq.push([\"_setAccount\", \"".length();
                int end =  content.indexOf("\"",begin);
                analyticAccount = content.substring(begin, end);
                
            } else {
                begin = content.indexOf("_uacct");
                if(begin > 0)
                {
                    begin+="_uacct".length();
                    analyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
                    begin=analyticAccount.lastIndexOf("-");
                    
                    analyticAccount = analyticAccount.substring(0, begin);
                } else {
                    Pattern p = Pattern.compile("UA-[0-9]+");

                    Matcher m = p.matcher(content);
                    while (m.find()) {
                        analyticAccount = m.group();
                    }
                }
            }
        }
        
        this.domain.setAnalytic(analyticAccount);
    }
    
    private void getGoogleAdsense() {
        int begin = content.indexOf("google_ad_client = \"");
        String adsAccount = "";
        if( begin > 0 )
        {
            begin+="google_ad_client = \"', '".length();
            int end =  content.indexOf("\"",begin);
            adsAccount = content.substring(begin, end);
        }
        
        this.domain.setAdsense(adsAccount);
    }
    
    private void getJavascriptLibrary() {
        List<String> jsLib = new ArrayList<String>();
        if(content.indexOf("jquery")>0)
            jsLib.add("jquery");
        
        if(content.indexOf("prototype.js")>0)
            jsLib.add("prototype");
        
        if(content.indexOf("extjs")>0)
            jsLib.add("extjs");
        
        if(content.indexOf("yui")>0)
            jsLib.add("yui");
        
        if(content.indexOf("dojo")>0)
            jsLib.add("dojo");
        
        if(content.indexOf("gwt")>0)
            jsLib.add("gwt");

        if(content.indexOf("easeljs")>0)
            jsLib.add("easeljs");
        
        this.domain.setJs(StringHelper.listToString(jsLib, ", "));
    }
    
    private void getWhois() {
        try {
            
            HTTPResponse response = (HTTPResponse) fWhois.get();
            
            Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
            
            this.domain.setWhois(new Text(doc.select("#whois-p1").html()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Error when get value whois");
        }
    }
    
    private void getIpAddress() {
        try {
            HTTPResponse response = (HTTPResponse) fIpAddress.get();
            
            this.domain.setIp(new String(response.getContent()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("ipAddress", "Can't get ipAddress for this domain " + domain.getDomain());
        }
    }
    
    private void getAlexaRank() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            String strAlexaRank = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
            int alexaRank = 0;
            
            try {
                if(strAlexaRank != null && !strAlexaRank.isEmpty())
                {
                    alexaRank = Integer.parseInt(strAlexaRank.replaceAll("[^0-9]+","")) ;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("alexaGlobeRank", "Can't get position alexa rank of domain " + domain.getDomain());
            }
            
            this.domain.setAlexa(alexaRank);
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getGooglePageRank() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleRank.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String googlePageRank = "0";
            String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
            try {
                if(google!= null && !google.isEmpty())
                {
                    googlePageRank = google;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
            }
            
            this.domain.setGoogle(Integer.parseInt(googlePageRank.replaceAll("[^0-9]+", "")));
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
        }
    }
    
    private void getAgeDomain() {
        try {
            HTTPResponse response = (HTTPResponse) fAgeDomain.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String age = doc.select("div[class=KonaBody]").select("p").text();
               String[] ageSplit = age.split("(approximately)");

               if(ageSplit.length ==2)
               {
                   this.domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
               }
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("age", "Can't get age of domain " + domain.getDomain());
        }
    }
    
    private void getInlink() {
    	try {
            Document doc = Jsoup.parse(this.content);
            Elements link = doc.select("a");
            
            ArrayList<String> listLink = new ArrayList<String>();
            String check = null;
            
            try {
    			ObjectifyService.register(Inlink.class);
    		} catch (Exception e) {
    			// TODO: handle exception
    		}
    		Objectify objectify = ObjectifyService.begin();
           
            for(int i=0;i<link.size();i++)
            {
                check = link.get(i).attr("rel"); 
                if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
                {
                    if( check != null && !check.equals("nofollow") || check == null)
                    {
                        Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
    
                        Matcher m = p.matcher(link.get(i).attr("href"));
                        while (m.find()) {
                            if(!listLink.contains(m.group())&&!Utils.fomatDomain(m.group()).equals(this.domain.getDomain()))
                            {
                            	if(Utils.checkDomain(Utils.fomatDomain(m.group())))
                            	{
	                                listLink.add(m.group());
	                                
	                                try {
	                                	Inlink inlink = objectify.get(new Key<Inlink>(Inlink.class, Utils.fomatDomain(m.group())));
	                                	inlink.addInLink(this.domain.getDomain());
	                                	objectify.put(inlink);
									} catch (Exception e) {
	//									e.printStackTrace();
										Inlink inlink = new Inlink();
										inlink.setDomain(Utils.fomatDomain(m.group()));
										inlink.addInLink(this.domain.getDomain());
										objectify.put(inlink);
									}
                            	}
                            }
                        }
                    }
                }
            }
            
            this.domain.setOutLink(listLink);
    	} catch (Exception e) {
        	e.printStackTrace();
            check.put("inlink", "Can't get inlink of domain " + domain.getDomain());
        }
    } 
}
